#!/usr/bin/perl -w
use DBI;
use warnings;
use MIME::Base64; 
use LWP::Protocol::https;
use LWP::UserAgent;
use Digest::MD5 qw(md5 md5_hex md5_base64);
use warnings;
use Time::Local;
use Date::Parse;
use Encode;
use strict;
use utf8;
binmode(STDIN, ':encoding(utf8)');
binmode(STDOUT, ':encoding(utf8)');
binmode(STDERR, ':encoding(utf8)');
&insert_news(1);

sub insert_news(){
    my $page= $_[0];
    my $driver="DBI:mysql";
	my $database="news";
	my $host="localhost";
	my $user="root";
	my $password="192510";
	my $db=DBI->connect("$driver:database=$database;host=$host;user=$user;password=$password");
	
	#得到前几天的时间
	my $current_time=$db->prepare("select UNIX_TIMESTAMP(date_sub(curdate(),interval 7 day))");
	$current_time->execute();
	my @current_time2=$current_time->fetchrow_array;
	my $current_time3=$current_time2[0];
	
	
	# my $sql= "set names utf8";
    # my $sth1=$db->prepare($sql);#准备
	# $sth1->execute(); 
	my $title;
	my @title_array;
    my $decoded_title;
	my @recs;
	my $sth=$db->prepare("select * from mygroup");
	$sth->execute();
	while(@recs=$sth->fetchrow_array){
	
	     $title=$recs[5];
		 @title_array=split(/\n/,$title); 
		 for(my $i=0;$i<=$#title_array;$i++){   #每一个关键字
		    $decoded_title= encode_base64($title_array[$i]); 
		    my $url="http://www.wooyun.org/searchbug.php?q=$decoded_title&showall=1&pNO=$page";
			
			my ($ua, $request , $response , $content);
			$ua = LWP::UserAgent->new;
			my $agentstr=&randomagent;
			$ua->agent('User-Agent'=>$agentstr);
			
			$request = HTTP::Request->new('GET',$url);
			$response = $ua->request($request);
			if($response->code ne '200'){
			
			
			}else{
					$content = $response->content;
					my $html=decode("utf-8",$content);
					#print $html;
					#去除<span class="red"></span> 便于拆分
					if ($html =~ /<span class="red">/) {
						 $html=~s/<span class="red">//g;
					} 
					if ($html =~ /<\/span>/) {
						 $html=~s/<\/span>//g;
					} 
					
					my @array=split(/<li><a target="_blank" class="atitle" href="/,$html);
				   
					my $array_num=scalar @array;
					
					for(my $i=1;$i<$array_num;$i++){
						#link
						my $href=$array[$i];
						my @href2=split(/">/,$href);
						my $href3=$href2[0];
						my $rel_href="http://www.wooyun.org".$href3;
						
						
						#标题
						my @title2=split(/<\/a>/,$href2[1]);
						my $title=$title2[0];
						
						
						#时间
						my @publishtime2=split(/提交日期：<a target="_blank" href="/,$href);
						#print $publishtime2[1]."\n\n\n\n\n\n\n";
						my @publishtime3=split(/<\/a> &nbsp;/,$publishtime2[1]);
						my @publishtime4=split(/">/,$publishtime3[0]);
						my $rel_publishtime=$publishtime4[1];
						#print $rel_publishtime."\n\n";
						
						#作者
						my @author=split(/作者：<a target="_blank"/,$href);
						my @author2=split(/<\/a>[\r\n\t]*<\/li>/,$author[1]);
						my @author3=split(/">/,$author2[0]);
						my $rel_author=$author3[1];
					
						#内容
						my $url=$rel_href;
						my ($ua, $request , $response , $content);
						$ua = LWP::UserAgent->new;
						$request = HTTP::Request->new('GET',$url);
						$response = $ua->request($request);
						$content = $response->content;
						my $html2=decode("utf-8",$content);
						
						my @content2=split(/<h3 class='wybug_type'>漏洞类型：/,$html2);
						my $content3=$content2[1];
						my @content4=split(/<\/h3>/,$content3);
						my $rel_content=$content4[0];
						$rel_content=~s/[\r\n\t]*//g;
						
						#加密
						my $m=$title.$rel_author;
						Encode::_utf8_off($m);
						my $md5=md5($m);
						
						my $web=3;
						
						my $bj_publishtime=str2time("$rel_publishtime GMT");
						if($bj_publishtime>=$current_time3){
							my $q_title=$title;
							$q_title=~s/\s//g;
							
							my $q_rel_href=$rel_href;
							$q_rel_href=~s/\s//g;
							
							my $q_rel_author=$rel_author;
							$q_rel_author=~s/\s//g;
							
							if($q_title ne"" and $q_rel_href ne"" and $q_rel_author ne""){
							
							  my $sql= "SET NAMES 'UTF8'";
							  my $sth=$db->prepare($sql);#准备
							  $sth = $db->prepare(q{insert ignore into news(title,web,author,publishtime,content,link,nmd5)VALUES(?,?,?,?,?,?,?)}) or die $db->errstr;
							  $sth->execute($title,$web,$rel_author,$rel_publishtime,$rel_content,$rel_href,$md5) or die $db->errstr;
							  
							}
						
						}
						
					}
			    
			}
	
		 }
		 
	}
	 sleep(1);
	 $page=$page+1;
	 &insert_news($page);
}


#模拟随机浏览器
sub randomagent
{
## FireFox
# Mozilla/5.0 (Windows NT 6.1; WOW64; rv:12.0) Gecko/20100101 Firefox/12.0
# Mozilla/5.0 (Windows NT 6.1; WOW64; rv:13.0) Gecko/20100101 Firefox/13.0.1
# Mozilla/5.0 (Windows NT 6.1; WOW64; rv:14.0) Gecko/20100101 Firefox/14.0.1
# Chrome
# Mozilla/5.0 (Windows; U; Windows NT 5.2) AppleWebKit/525.13 (KHTML, like Gecko) Chrome/0.2.149.27 Safari/525.13
# Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.168 Safari/535.19 QIHU 360EE
# Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.57 Safari/536.11
# Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.89 Safari/537.1
##Safari
# Mozilla/5.0 (Windows; U; Windows NT 5.2) AppleWebKit/525.13 (KHTML, like Gecko) Version/3.1 Safari/525.13
##Sogou
# Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.84 Safari/535.11 SE 2.X MetaSr 1.0
##Baidu
# Mozilla/5.0 (Windows; U; Windows NT 6.1; zh_CN) AppleWebKit/534.7 (KHTML, like Gecko) Chrome/18.0 BIDUBrowser/2.6 Safari/534.7
##QQ
# Mozilla/5.0 (Windows NT 6.1) AppleWebKit/536.7 (KHTML, like Gecko) Chrome/20.0.1099.0 Safari/536.7 QQBrowser/6.14.15138.201
##Maxthon
# Mozilla/5.0 (Windows NT 6.1) AppleWebKit/535.12 (KHTML, like Gecko) Maxthon/3.4.2.3000 Chrome/18.0.966.0 Safari/535.12
##LBBROWSER
# Mozilla/5.0 (Windows NT 6.1) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.84 Safari/535.11 LBBROWSER
##TheWorld
# Mozilla/5.0 (Windows NT 6.1) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.79 Safari/535.11 QIHU THEWORLD
	my @agent_conf=('Mozilla/5.0 (Windows NT 6.1; WOW64; rv:12.0) Gecko/20100101 Firefox/12.0',
	'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:13.0) Gecko/20100101 Firefox/13.0.1',
	'Mozilla/5.0 (Windows; U; Windows NT 5.2) AppleWebKit/525.13 (KHTML, like Gecko) Chrome/0.2.149.27 Safari/525.13',
	'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.168 Safari/535.19 QIHU 360EE',
	'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.57 Safari/536.11',
	'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.89 Safari/537.1',
	'Mozilla/5.0 (Windows; U; Windows NT 5.2) AppleWebKit/525.13 (KHTML, like Gecko) Version/3.1 Safari/525.13',
	'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.84 Safari/535.11 SE 2.X MetaSr 1.0',
	'Mozilla/5.0 (Windows; U; Windows NT 6.1; zh_CN) AppleWebKit/534.7 (KHTML, like Gecko) Chrome/18.0 BIDUBrowser/2.6 Safari/534.7',
	'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/535.12 (KHTML, like Gecko) Maxthon/3.4.2.3000 Chrome/18.0.966.0 Safari/535.12',
	'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.84 Safari/535.11 LBBROWSER',
	'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.79 Safari/535.11 QIHU THEWORLD');
	my $int=int(rand(12));
	return $agent_conf[$int];
}



